/**
 * Copyright (C) 2024 AIDC-AI
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.alibaba.langengine.core.tokenizers;

import com.alibaba.langengine.core.jtokkit.api.EncodingType;
import lombok.Data;

import java.util.List;

/**
 * Tokenizer基类
 *
 * @author xiaoxuan.lp
 */
@Data
public abstract class Tokenizer {

    /**
     * 编码
     *
     * @param text
     * @return
     */
    public List<Integer> encode(String text) {
        return TikTokensUtil.encode(getEncoding(), text);
    }

    /**
     * 解码
     *
     * @param encodes
     * @return
     */
    public String decode(List<Integer> encodes) {
        return TikTokensUtil.decode(getEncoding(), encodes);
    }

    /**
     * 统计token个数
     *
     * @param text
     * @return
     */
    public int getTokenCount(String text) {
        return TikTokensUtil.tokens(getEncoding(), text);
    }

    /**
     * 获取编码器
     *
     * @return
     */
    public abstract EncodingType getEncoding();
}
